/**
 * @author Fang Lu, fanglupku@gmail.com
 * 2011-5-12
 */
package com.autotag;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import java.util.Set;

import com.autotag.entity.NPEntity;
import com.autotag.entity.TagEntity;
import com.autotag.nlp.GramStat;
import com.autotag.nlp.WordStemmer;

import fanglu.util.FileDealer;
import fanglu.util.StringDealer;

import sei.pku.wsdl_analyzer.DefineType;
import sei.pku.wsdl_analyzer.Message;
import sei.pku.wsdl_analyzer.Operation;
import sei.pku.wsdl_analyzer.WSDLFile;
import sei.pku.wsdl_analyzer.WSDLFileAnalyzer;

public class FunctionalityTagGenerator {
	public static final String CONTENT_SEPERATOR = "|";
	public static void generateInfoFile(File dirFile, File targetFile){
		File[] files = dirFile.listFiles();
		try {
			FileOutputStream fos = new FileOutputStream(targetFile);
			OutputStreamWriter osw = new OutputStreamWriter(fos);
			BufferedWriter bw = new BufferedWriter(osw);
			
			for (int i=0; i<files.length; ++i){
				File thisFile = files[i];
				bw.write("#");
				bw.write(thisFile.getName());
				bw.write("\n");
				WSDLFileAnalyzer wsdlFileAnalyzer = new WSDLFileAnalyzer();
				try {
					WSDLFile wFile = wsdlFileAnalyzer.analyzeWSDL(thisFile.getAbsolutePath());
					
					ArrayList<String> level1StringList = new ArrayList<String>();
					ArrayList<String> level2StringList = new ArrayList<String>();
					ArrayList<String> level3StringList = new ArrayList<String>();
					ArrayList<String> level4StringList = new ArrayList<String>();
					ArrayList<String> level5StringList = new ArrayList<String>();
					
					//level 1, get name
					ArrayList<String> tokenList = TokenExtractor.extractTokenFromCamelString(wFile.name);
					String serviceNameStr = TokenExtractor.generateString(tokenList, " ");
					level1StringList.add(serviceNameStr);
					
					//level 2, get op name
					ArrayList<Operation> opList = wFile.operations;
					for (int j=0; j<opList.size(); ++j){
						ArrayList<String> opTokenList = TokenExtractor.extractTokenFromCamelString(opList.get(j).operationName);
						String opNameStr = TokenExtractor.generateString(opTokenList, " ");
						if (opNameStr.length() > 0){
							level2StringList.add(opNameStr);
						}
					}
					
					//level 3, get message name
					ArrayList<Message> mList = wFile.messages;
					for (int j=0; j<mList.size(); ++j){
						ArrayList<String> mTokenList = TokenExtractor.extractTokenFromCamelString(mList.get(j).name);
						String mNameStr = TokenExtractor.generateString(mTokenList, " ");
						if (mNameStr.length() > 0){
							level3StringList.add(mNameStr);
						}
					}
					
					//level 4,5 get type name and element
					ArrayList<DefineType> tList = wFile.types;
					for (int j=0; j<tList.size(); ++j){
						ArrayList<String> tTokenList = TokenExtractor.extractTokenFromCamelString(tList.get(j).name);
						String tNameStr = TokenExtractor.generateString(tTokenList, " ");
						if (tNameStr.length() > 0){
							level4StringList.add(tNameStr);
						}
						Map<String, String> elementMap = tList.get(j).elements;
						Iterator<String> itor = elementMap.keySet().iterator();
						while (itor.hasNext()){
							String thisKey = itor.next();
							ArrayList<String> eTokenList = TokenExtractor.extractTokenFromCamelString(thisKey);
							String eNameStr = TokenExtractor.generateString(eTokenList, " ");
							if (eNameStr.length() > 0){
								level5StringList.add(eNameStr);
							}
							String thisValue = elementMap.get(thisKey);
							eTokenList = TokenExtractor.extractTokenFromCamelString(thisValue);
							String eValueStr = TokenExtractor.generateString(eTokenList, " ");
							if (eValueStr.length() > 0){
								level5StringList.add(eValueStr);
							}
						}
					}
					bw.write("1:");
					for (int j=0; j<level1StringList.size(); ++j){
						bw.write(level1StringList.get(j));
						bw.write(CONTENT_SEPERATOR);
					}
					bw.write("\n");
		
					bw.write("2:");
					for (int j=0; j<level2StringList.size(); ++j){
						bw.write(level2StringList.get(j));
						bw.write(CONTENT_SEPERATOR);
					}
					bw.write("\n");
					
					bw.write("3:");
					for (int j=0; j<level3StringList.size(); ++j){
						bw.write(level3StringList.get(j));
						bw.write(CONTENT_SEPERATOR);
					}
					bw.write("\n");
					
					bw.write("4:");
					for (int j=0; j<level4StringList.size(); ++j){
						bw.write(level4StringList.get(j));
						bw.write(CONTENT_SEPERATOR);
					}
					bw.write("\n");
					
					bw.write("5:");
					for (int j=0; j<level5StringList.size(); ++j){
						bw.write(level5StringList.get(j));
						bw.write(CONTENT_SEPERATOR);
					}
					bw.write("\n");
					System.out.println(i + ",ok");
					
				} catch (Exception e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				
			}
			
			bw.close();
			osw.close();
			fos.close();
			
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	
	public static ArrayList<TagEntity> getTagEntityList(File thisWSDLFile){
		ArrayList<TagEntity> resultList = new ArrayList<TagEntity>();
		
		WSDLFileAnalyzer wsdlFileAnalyzer = new WSDLFileAnalyzer();
		try {
			WSDLFile wFile = wsdlFileAnalyzer.analyzeWSDL(thisWSDLFile.getAbsolutePath());
			//first, get service name, level 1 tag(s)
//			System.out.println("service name:" + wFile.name);
			ArrayList<String> tokenList = TokenExtractor.extractTokenFromCamelString(wFile.name);
			for (int i=0; i<tokenList.size(); ++i){
				TagEntity thisTagEntity = new TagEntity(tokenList.get(i), 1, 0);
				resultList.add(thisTagEntity);
			}
			
			//second, get operation name, level 2 tag(s)
			ArrayList<Operation> opList = wFile.operations;
			for (int i=0; i<opList.size(); ++i){
				ArrayList<String> opTokenList = TokenExtractor.extractTokenFromCamelString(opList.get(i).operationName);
				for (int j=0; j<opTokenList.size(); ++j){
					TagEntity thisTagEntity = new TagEntity(opTokenList.get(j), 2, 0);
					resultList.add(thisTagEntity);
				}
			}
			
			//third, get message name and part, level 3 tag(s)
			ArrayList<Message> mList = wFile.messages;
			for (int i=0; i<mList.size(); ++i){
				ArrayList<String> mTokenList = TokenExtractor.extractTokenFromCamelString(mList.get(i).name);
				for (int j=0; j<mTokenList.size(); ++j){
					TagEntity thisTagEntity = new TagEntity(mTokenList.get(j), 3, 0);
					resultList.add(thisTagEntity);
				}
			}
			
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		return resultList;
	}
	public static void generateTagByTFIDF(String tagFile, 
			HashMap<String, HashMap<String, Integer>> stemMapUUID, 
			HashMap<String, HashMap<String, Integer>> UUIDMapStem,
			HashMap<String, HashMap<String, HashSet<Integer>>> UUIDMapStemLevel,
			HashMap<String, ArrayList<TagEntity>> UUIDMapTagEntityList, 
			HashMap<String, HashMap<String, NPEntity>> UUIDMapNP){
		Set<String> UUIDSet = UUIDMapStem.keySet();
		Iterator<String> UUID_itor = UUIDSet.iterator();
		int documentCount = UUIDMapStem.size();
		while (UUID_itor.hasNext()){
			ArrayList<TagEntity> tagList = new ArrayList<TagEntity>();
			String thisUUID = UUID_itor.next();
			int maxFreq = 1;
			HashMap<String, Integer> stemMapCount = UUIDMapStem.get(thisUUID);
			Iterator<String> stem_itor = stemMapCount.keySet().iterator();
			while (stem_itor.hasNext()){
				String thisStem = stem_itor.next();
				int thisFreq = stemMapCount.get(thisStem);
				if (thisFreq > maxFreq){
					maxFreq = thisFreq;
				}
			}
			HashMap<String, HashSet<Integer>> stemMapLevel = UUIDMapStemLevel.get(thisUUID);
			HashMap<String, NPEntity> thisNPMap = UUIDMapNP.get(thisUUID);
			stem_itor = stemMapCount.keySet().iterator();
			while (stem_itor.hasNext()){
				String thisStem = stem_itor.next();
				int thisFreq = stemMapCount.get(thisStem);
				double tf = 0.5 + 0.5 * ((double) thisFreq) / ((double) maxFreq);
				int df = 0;
				try{
					df = stemMapUUID.get(thisStem).size();
				}catch (Exception e) {
					e.printStackTrace();
				}
				double idf = Math.log(((double) documentCount) / df * 0.5 + 0.6);
				HashSet<Integer> thisLevelSet = stemMapLevel.get(thisStem);
				double minLevel = 100;
				for (Integer thisLevel : thisLevelSet) {
					if (((double) thisLevel) < minLevel){
						minLevel = thisLevel;
					}
				}
				int tokenNum = NPEntity.getTokenNum(thisStem, " ");
				if (tokenNum > 1){
					int this_appear = 0;
					Iterator<String> thisStem_itor = thisNPMap.keySet().iterator();
					while (thisStem_itor.hasNext()){
						String tempStem = thisStem_itor.next();
						if (tempStem.indexOf(thisStem) >= 0){
							++ this_appear;
						}
					}
					if (this_appear <= 1 
							&& !thisNPMap.get(thisStem).getLevelMapCount().containsKey(1)
							){
						continue;
					}
					tf = 0.5 + 0.5 * ((double) this_appear) / ((double) maxFreq);
					idf = Math.log(((double) documentCount) / (df+1)*0.5 + 0.6);
				}
				double np_bonus = 0.0;
				NPEntity thisNP = thisNPMap.get(thisStem);
				if (thisNP != null){
					np_bonus = 1;
				}
				//add for test
//				else{
//					if (minLevel <= 2){
//						continue;
//					}
//				}
				try{
					Double.parseDouble(thisStem);
					continue;
				}catch (Exception e) {
					// TODO: handle exception
				}
				
				
				double weight =  0.5 * 1.0 / (minLevel * minLevel) + 0.3 * np_bonus + 0.2 * tf * idf;
				
				TagEntity thisTag = new TagEntity(thisStem, 1, weight);
				tagList.add(thisTag);
			}
			Collections.sort(tagList);
			UUIDMapTagEntityList.put(thisUUID, tagList);
		}
		
		File file = new File(tagFile);
		
		try {
//			if (!file.exists()){
//				file.createNewFile();
//			}
			FileOutputStream fos = new FileOutputStream(file);
			OutputStreamWriter osw = new OutputStreamWriter(fos);
			BufferedWriter bw  = new BufferedWriter(osw);
			
			Iterator<String> itor = UUIDMapTagEntityList.keySet().iterator();
			while (itor.hasNext()){
				String thisUUID = itor.next();
				ArrayList<TagEntity> thisTagList = UUIDMapTagEntityList.get(thisUUID);
				bw.write("#");
				bw.write(thisUUID);
				bw.write("\n");
				for (int i=0; i<thisTagList.size(); ++i){
					if (WSDLStopWord.isStopWord(thisTagList.get(i).getName())){
						continue;
					}
					bw.write(thisTagList.get(i).getName() + "," + thisTagList.get(i).getWeight() + "\n");
				}
			}
			
			bw.close();
			osw.close();
			fos.close();
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	public static void selectTagFromCandidate(File srcFile, File targetFile, int total, int max,
			double threshold, int max_tags_per_service){
		Random random = new Random(System.currentTimeMillis());
		try {
			BufferedWriter bw = new BufferedWriter(new FileWriter(targetFile));
			String content = FileDealer.getFileContent(srcFile.getAbsolutePath());
			BufferedReader br = new BufferedReader(new StringReader(content));
			String oneLine = br.readLine();
			boolean isWrite = false;
			int ready = 0;
			ArrayList<String> thisTagList = new ArrayList<String>();
			String thisUUID = "";
			while (oneLine != null){
				oneLine = oneLine.trim();
				if (oneLine.startsWith("#")){
					if (ready == total){
						break;
					}
					int thisInt = random.nextInt() % max;
					if (thisInt < total){
						isWrite = true;
						if (thisUUID.length() > 0){
							for (int i=0; i<thisTagList.size(); ++i){
								bw.write(thisTagList.get(i));
								bw.write("\n");
							}
						}
						thisTagList.clear();
						thisUUID = oneLine;
						++ ready;
						bw.write(oneLine + "\n");
					}else{
						isWrite = false;
					}
				}else{
					if (isWrite){
						String[] sa = oneLine.split(",");
						if (sa.length != 2){
							
						}else{
							try{
								double thisvalue = Double.parseDouble(sa[1]);
								boolean isNum = false;
								String thisTag = sa[0];
								try{
									Double.parseDouble(thisTag);
									isNum = true;
								}catch (Exception e) {
								}
								if (thisvalue < threshold
										|| WSDLStopWord.isStopWord(thisTag) == true
										|| sa[0].length() < 2
										|| isNum == true
										|| sa[0].contains("%")){
									
								}else{
									if (max_tags_per_service >= thisTagList.size()){
										int reduadant_index = -1;
										for (int i=0; i<thisTagList.size(); ++ i){
											
											if (StringDealer.isContainingByToken(thisTag, thisTagList.get(i))){
												reduadant_index = i;
												break;
											}else if (StringDealer.isContainedByToken(thisTag, thisTagList.get(i))){
												reduadant_index = -2;
												break;
											}
										}
										if (reduadant_index == -2){
											
										}else if (reduadant_index > -1){
											thisTagList.set(reduadant_index, thisTag);
										}else{
											thisTagList.add(thisTag);
										}
									}

								}
							}catch (Exception e) {
								e.printStackTrace();
							}
						}
					}
				}
				oneLine = br.readLine();
			}
			bw.close();
			br.close();
			
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	
	public static void statTag(File tagFile, double threshold){
		
		try {
			String content = FileDealer.getFileContent(tagFile.getAbsolutePath());
			BufferedReader br = new BufferedReader(new StringReader(content));
			String oneLine = br.readLine();
			String thisUUID = "";
			HashMap<String, HashSet<String>> uuidMapTagSet = new HashMap<String, HashSet<String>>();
			HashMap<String, HashSet<String>> tagMapUUIDSet = new HashMap<String, HashSet<String>>();
			HashSet<String> thisTagSet = new HashSet<String>();
			HashSet<String> thisUUIDSet = new HashSet<String>();
			while (oneLine != null){
				oneLine = oneLine.trim();
				if (oneLine.startsWith("#")){
					thisUUID = oneLine;
					thisTagSet = new HashSet<String>();
					uuidMapTagSet.put(thisUUID, thisTagSet);
				}else{
					String[] sa = oneLine.split(",");
					if (sa.length  != 2) {
						
					}else{
						sa[0] = WordStemmer.getStem(sa[0]);
						try{
							double thisvalue = Double.parseDouble(sa[1]);
							if (thisvalue < threshold
									|| WSDLStopWord.isStopWord(sa[0]) == true
									|| sa[0].length() < 2){
								
							}else{
								thisUUIDSet = tagMapUUIDSet.get(sa[0]);
								if (thisUUIDSet == null){
									thisUUIDSet = new HashSet<String>();
									tagMapUUIDSet.put(sa[0], thisUUIDSet);
								}
								thisUUIDSet.add(thisUUID);
								thisTagSet.add(sa[0]);
							}
						}catch (Exception e) {
							e.printStackTrace();
						}
						
					}
					
				}
				oneLine = br.readLine();
			}
			br.close();
//			Iterator<String> itor_term = tagMapUUIDSet.keySet().iterator();
//			while(itor_term.hasNext()){
//				String thisTerm = itor_term.next();
//				System.out.println(thisTerm + "," + tagMapUUIDSet.get(thisTerm).size());
//			}
			
			Iterator<String> itor_uuid = uuidMapTagSet.keySet().iterator();
			while(itor_uuid.hasNext()){
				thisUUID = itor_uuid.next();
				System.out.println(thisUUID + "," + uuidMapTagSet.get(thisUUID).size());
			}
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	
	public static void main(String args[]){
		try {
//			String basePath = "C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\WSDL_FILE\\";
//			String filePath = "000cfe33-460f-4454-92f5-d9c1acf2b84a.html";
//			File file = new File(basePath + filePath);
//			
//			ArrayList<TagEntity> tagList = getTagEntityList(file);
//			HashMap<String, TagEntity> tagNameMapLevel = new HashMap<String, TagEntity>();
//			for (int i=0; i<tagList.size(); ++i){
//				TagEntity thisTagEntity = tagList.get(i);
//				if (!tagNameMapLevel.containsKey(thisTagEntity.getName())){
//					tagNameMapLevel.put(thisTagEntity.getName(), thisTagEntity);
//					System.out.println(thisTagEntity.getName() + "," + thisTagEntity.getLevel());
//				}
//			}
			
			//part 2, generate info file
//			String basePath = "C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\";
//			File dirFile = new File(basePath + "WSDL_FILE");
//			File resultFile = new File(basePath + "result\\result.txt");
//			generateInfoFile(dirFile, resultFile);
//			
			//part 3
			String tagFile = "C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\tag.txt";
//			File stemMapUUIDFile = new File("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\stem_uuid.txt");
//			File UUIDMapStemFile = new File("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\uuid_stem.txt");
//			HashMap<String, HashMap<String, Integer>> stemMapUUID = GramStat.initGramFromMapFile(stemMapUUIDFile);
//			HashMap<String, HashMap<String, Integer>> UUIDMapStem = GramStat.initGramFromMapFile(UUIDMapStemFile);
//			Iterator<String> itor_UUID = UUIDMapStem.keySet().iterator();
//			while (itor_UUID.hasNext()){
//				String thisUUID = itor_UUID.next();
//				HashMap<String, Integer> map = UUIDMapStem.get(thisUUID);
//				if (map.keySet().size() == 0){
//					System.out.println(thisUUID + "/" + map.keySet().size());
//				}
//				
//			}
			
//			HashMap<String, HashMap<String, Integer>> stemMapUUID = new HashMap<String, HashMap<String,Integer>>();
//			HashMap<String, HashMap<String, Integer>> UUIDMapStem = new HashMap<String, HashMap<String,Integer>>();
//			HashMap<String, HashMap<String, HashSet<Integer>>> UUIDMapStemLevel = new HashMap<String, HashMap<String,HashSet<Integer>>>();
//			HashMap<String, HashMap<String, NPEntity>> UUIDMapNP = new HashMap<String, HashMap<String,NPEntity>>();
			
			//part4
			HashMap<String, HashMap<String, Integer>> stemMapUUID = 
				(HashMap<String, HashMap<String, Integer>>) GramStat.readObjectFromFile("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\stem_uuid.txt");
			HashMap<String, HashMap<String, Integer>> UUIDMapStem = 
				(HashMap<String, HashMap<String, Integer>>) GramStat.readObjectFromFile("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\uuid_stem.txt");
			HashMap<String, HashMap<String, HashSet<Integer>>> UUIDMapStemLevel = 
				(HashMap<String, HashMap<String, HashSet<Integer>>>) GramStat.readObjectFromFile("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\uuid_stemlevel.txt");
			HashMap<String, HashMap<String, NPEntity>> UUIDMapNP = 
				(HashMap<String, HashMap<String, NPEntity>>) GramStat.readObjectFromFile("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\uuid_np.txt");
			
			HashMap<String, ArrayList<TagEntity>> UUIDMapTagEntityList = new HashMap<String, ArrayList<TagEntity>>();
			generateTagByTFIDF(tagFile, stemMapUUID, UUIDMapStem, UUIDMapStemLevel, UUIDMapTagEntityList, UUIDMapNP);

			File srcFile = new File("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\tag.txt");
			File targetFile = new File("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\select_tag.txt");
			selectTagFromCandidate(srcFile, targetFile, Integer.MAX_VALUE, Integer.MAX_VALUE, 0.5, 20);
			

			//part 5			
//			statUsefulToken();
//			statTag(srcFile, 2.0);
			
			
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	public static void statUsefulToken(){
		HashMap<String, HashMap<String, Integer>> stemMapUUID = 
			(HashMap<String, HashMap<String, Integer>>) GramStat.readObjectFromFile("C:\\Users\\ftc\\Desktop\\ICSOC\\WSDL_FILE\\result\\stem_uuid.txt");
		
		System.out.println("term size:" + stemMapUUID.keySet().size());
		Iterator<String> thisStem = stemMapUUID.keySet().iterator();
		int total = 0;
		int single = 0;
		int count_useful = 0;
		int count_un = 0;
		while (thisStem.hasNext()){
			String term = thisStem.next();
			if (term.indexOf(" ") > 0){
				
			}else{
				++ single;
				if (WordStemmer.rootMap.containsKey(term)){
//					UUIDSet.addAll(stemMapUUID.get(term).keySet());
					total ++;
					HashMap<String, Integer> uuidMapCount = stemMapUUID.get(term);
					Iterator<String> uuid = uuidMapCount.keySet().iterator();
					while (uuid.hasNext()){
						String thisUUID = uuid.next();
						count_useful += uuidMapCount.get(thisUUID);
					}
				}else{
					System.out.println(term);
					HashMap<String, Integer> uuidMapCount = stemMapUUID.get(term);
					Iterator<String> uuid = uuidMapCount.keySet().iterator();
					while (uuid.hasNext()){
						String thisUUID = uuid.next();
						count_un += uuidMapCount.get(thisUUID);
					}
				}
			}
			
		}
		
		System.out.println("useful term size:" + total);
		System.out.println("one gram:" + single);
		System.out.println("used size:" + count_useful);
		System.out.println("noused uuid size:" + count_un);
	}
}
